昨天介紹過如何迭代DeepStream的Metadata，今天我們來細看每種Metadata的成員，對於日後要調整Metadata會很有幫助

NvDsBatchMeta

typedef struct _NvDsBatchMeta {
  NvDsBaseMeta base_meta;
  /** Holds the maximum number of frames in the batch. */
  guint max_frames_in_batch;
  /** Holds the number of frames now in the batch. */
  guint num_frames_in_batch;
  /** Holds a pointer to a pool of pointers of type @ref NvDsFrameMeta, representing a pool of frame metas. */
  NvDsMetaPool *frame_meta_pool;
  /** Holds a pointer to a pool of pointers of type NvDsObjMeta, representing a pool of object metas. */
  NvDsMetaPool *obj_meta_pool;
  /** Holds a pointer to a pool of pointers of type @ref NvDsClassifierMeta, representing a pool of classifier metas. */
  NvDsMetaPool *classifier_meta_pool;
  /** Holds a pointer to a pool of pointers of type @ref NvDsDisplayMeta, representing a pool of display metas. */
  NvDsMetaPool *display_meta_pool;
  /** Holds a pointer to a pool of pointers of type @ref NvDsUserMeta, representing a pool of user metas. */
  NvDsMetaPool *user_meta_pool;
  /** Holds a pointer to a pool of pointers of type @ref NvDsLabelInfo, representing a pool of label metas. */
  NvDsMetaPool *label_info_meta_pool;
  /** Holds a pointer to a list of pointers of type NvDsFrameMeta or NvDsAudioFrameMeta (when the batch represent audio batch), representing frame metas used in the current batch. */
  NvDsFrameMetaList *frame_meta_list;
  /** Holds a pointer to a list of pointers of type NvDsUserMeta, representing user metas in the current batch. */
  NvDsUserMetaList *batch_user_meta_list;
  /** Holds a lock to be set before accessing metadata to avoid simultaneous update by multiple components. */
  GRecMutex meta_mutex;
  /** Holds an array of user-specific batch information. */
  gint64 misc_batch_info[MAX_USER_FIELDS];
  /** For internal use. */
  gint64 reserved[MAX_RESERVED_FIELDS];
} NvDsBatchMeta;

先來看一下裡面有好幾個欄位是NvDsMetaPool，DeepStream為了應付高吞吐量，會使用Pool來分配MetaData需要的空間，分配出去的MetaData在GstBuffer被釋放後返回Pool內供新的GstBuffer使用。另外，在我的開發經歷中有因為Bug意外觸發Pool用盡的情況，因此Pool是有空間上限的，但數量還是非常充足的。

NvDsBatchMeta通常就是迭代frame_meta_list和讀取num_frames_in_batch，其他欄位不會使用到

NvDsFrameMeta

/**
 * Holds metadata for a frame in a batch.
 */
typedef struct _NvDsFrameMeta {
  /** Holds the base metadata for the frame. */
  NvDsBaseMeta base_meta;
  /** Holds the pad or port index of the Gst-streammux plugin for the frame in the batch. */
  guint pad_index;
  /** Holds the location of the frame in the batch. The frame's @ref NvBufSurfaceParams are at index @a batch_id in the @a surfaceList array of @ref NvBufSurface. */
  guint batch_id;
  /** Holds the current frame number of the source. */
  gint frame_num;
  /** Holds the presentation timestamp (PTS) of the frame. */
  guint64 buf_pts;
  /** Holds the ntp timestamp.�| */
  guint64 ntp_timestamp;
  /** Holds the source IDof the frame in the batch, e.g. the camera ID. It need not be in sequential order. */
  guint source_id;
  /** Holds the number of surfaces in the frame, required in case of multiple surfaces in the frame. */
  gint num_surfaces_per_frame;
  /* Holds the width of the frame at input to Gst-streammux. */
  guint source_frame_width;
  /* Holds the height of the frame at input to Gst-streammux. */
  guint source_frame_height;
  /* Holds the surface type of the subframe, required in case of multiple surfaces in the frame. */
  guint surface_type;
  /* Holds the surface index of tje subframe, required in case of multiple surfaces in the frame. */
  guint surface_index;
  /** Holds the number of object meta elements attached to current frame. */
  guint num_obj_meta;
  /** Holds a Boolean indicating whether inference is performed on the frame. */
  gboolean bInferDone;
  /** Holds a pointer to a list of pointers of type @ref NvDsObjectMeta in use for the frame. */
  NvDsObjectMetaList *obj_meta_list;
  /** Holds a pointer to a list of pointers of type @ref NvDsDisplayMeta in use for the frame. */
  NvDisplayMetaList *display_meta_list;
  /** Holds a pointer to a list of pointers of type @ref NvDsUserMeta in use for the frame. */
  NvDsUserMetaList *frame_user_meta_list;
  /** Holds additional user-defined frame information. */
  gint64 misc_frame_info[MAX_USER_FIELDS];
  /* Holds the width of the frame at output of Gst-streammux. */
  guint pipeline_width;
  /* Holds the height of the frame at output of Gst-streammux. */
  guint pipeline_height;
  NvDsSensorInfo_meta sensorInfo_meta;
  /** For internal use. */
  gint64 reserved[MAX_RESERVED_FIELDS];
} NvDsFrameMeta;

要取得影像的數據就使用batch_id去NvBufSurface內找，source_id是影像來源的識別符，兩者容易搞混，使用時要留意

source_frame_width、source_frame_height是影像來源的尺寸，pipeline_width和pipeline_height是nvstreammux設定的尺寸

最後是這個看似不起眼的bInferDone，只有在設定為TRUE的時候nvtracker才會工作，否則它會把所有MetaData移除。

NvDsObjectMeta

/**
 * Holds metadata for an object in the frame.
 */
typedef struct _NvDsObjectMeta {
  NvDsBaseMeta base_meta;
  /** Holds a pointer to the parent @ref NvDsObjectMeta. Set to NULL if no parent exists. */
  struct _NvDsObjectMeta *parent;
  /** Holds a unique component ID that identifies the metadata
   in this structure. */
  gint unique_component_id;
  /** Holds the index of the object class inferred by the primary detector/classifier. */
  gint class_id;
  /** Holds a unique ID for tracking the object. @ref UNTRACKED_OBJECT_ID indicates that the object has not been tracked. */
  guint64 object_id;
  /** Holds a structure containing bounding box parameters of the object when detected by detector. */
  NvDsComp_BboxInfo detector_bbox_info;
  /** Holds a structure containing bounding box coordinates of the object when processed by tracker. */
  NvDsComp_BboxInfo tracker_bbox_info;
  /** Holds a structure containing positional parameters of the object processed by the last component that updates it in the pipeline. e.g. If the tracker component is after the detector component in the pipeline then positinal parameters are from tracker component. Positional parameters are clipped so that they do not fall outside frame boundary. Can also be used to overlay borders or semi-transparent boxes on objects. @see NvOSD_RectParams. */
  NvOSD_RectParams rect_params;
  /** Holds mask parameters for the object. This mask is overlayed on object @see NvOSD_MaskParams. */
  NvOSD_MaskParams mask_params;
  /** Holds text describing the object. This text can be overlayed on the standard text that identifies the object. @see NvOSD_TextParams. */
  NvOSD_TextParams text_params;
  /** Holds a string describing the class of the detected object. */
  gchar obj_label[MAX_LABEL_SIZE];
  /** Holds a pointer to a list of pointers of type @ref NvDsClassifierMeta. */
  NvDsClassifierMetaList *classifier_meta_list;
  /** Holds a pointer to a list of pointers of type @ref NvDsUserMeta. */
  NvDsUserMetaList *obj_user_meta_list;
  /** Holds additional user-defined object information. */
  gint64 misc_obj_info[MAX_USER_FIELDS];
  /** For internal use. */
  gint64 reserved[MAX_RESERVED_FIELDS];
}NvDsObjectMeta;

unique_component_id代表這個MetaData是從哪一個nvinfer產生的，如果我們要開發自己的元件跟DeepStream元件配合使用，這個欄位要記得填上

class_id則是物件屬於哪一個類別，基本上物件偵測的模型輸出都有分類結果，直接套上0-based整數就好

object_id則是nvtracker產生的ID，如果沒加入nvtracker，預設是UNTRACKED_OBJECT_ID，數值是0xFFFFFFFFFFFFFFFF

obj_label和text_params.display_text這兩個很容易混淆，obj_label只是儲存一個標籤讓你知道這個物件分類是什麼，真正經過nvdsosd畫出來的是text_params.display_text

parent則是應用在多個物件偵測模型串聯使用。當元件有設定上層元件時時(operate-on-gie-id)，他會把上層產生的Bounding Box擷取ROI當S做輸入去偵測不同的物件，完成後SGIE會把產生的NvDsObjectMeta的parent設定為上層的NvDsObjectMeta。舉個容易理解的例子，第一個元件偵測「人」，第二個元件在基於「人」的ROI去偵測「帽子」、「手錶」等其他不同的物體，而這些物體就屬於這個「人」

NvDsClassifierMeta和NvDsLabelInfo

/**
 * Holds classifier metadata for an object.
 */
typedef struct _NvDsClassifierMeta {
  NvDsBaseMeta base_meta;
  /** Holds the number of outputs/labels produced by the classifier. */
  guint num_labels;
  /** Holds a unique component ID for the classifier metadata. */
  gint unique_component_id;
  /** Holds a pointer to a list of pointers of type @ref NvDsLabelInfo. */
  NvDsLabelInfoList *label_info_list;
  /** Holds the type of the classifier. */
  const gchar *classifier_type;
} NvDsClassifierMeta;

/**
 * Holds label metadata for the classifier.
 */
typedef struct _NvDsLabelInfo {
  NvDsBaseMeta base_meta;
  /** Holds the number of classes of the given label. */
  guint num_classes;
  /** Holds an string describing the label of the classified object. */
  gchar result_label[MAX_LABEL_SIZE];
  /** Holds a pointer to the result label if its length exceeds MAX_LABEL_SIZE bytes. */
  gchar *pResult_label;
  /** Holds the class UD of the best result. */
  guint result_class_id;
  /** Holds the label ID in case there are multiple label classifiers. */
  guint label_id;
  /** Holds the probability of best result. */
  gfloat result_prob;
} NvDsLabelInfo;

NvDsClassifierMeta是SGIE根據PGIE產出的Bounding Box去判斷這個物體是什麼，官方的例子是PGIE找出車子，SGIE負責分類車子的廠牌。

unique_component_id這邊是SGIE的ID，而label_info_list在官方的解釋是一個Classifier可以有能產出多個分類結果(看模型而定)。NvDsLabelInfo則是儲存分類結果的地方，nvinfer預設會從這邊的資料去擴展display_text的內容，例如物體是car，分類是toyota，最終的display_text就會變成car toyota。

label_id、num_classes跟result_class_id比較難以理解一點，這個是Classifer的Label檔案編排方式，我貼上官方的範例

acura;audi;bmw;chevrolet;chrysler;dodge;ford;gmc;honda;hyundai;infiniti;jeep;kia;lexus;mazda;mercedes;nissan;subaru;toyota;volkswagen
# 第2個label
# 第3個label

官方的只有一種label，後面的文字是我補充的，所以label_id就只會是0，而num_classes就是看第0行用;分隔了幾個類別，result_class_id就是這些類別的index，例如bmw就是2

NvDsBaseMeta和NvDsUserMeta

/**
 * \brief  Defines the type of a callback to copy metadata.
 *
 * The callback is passed pointers to source metadata. It allocates the
 * required memory, copies the content from the source metadata and returns
 * a pointer to the destination.
 *
 * @param[in]  data         A pointer to the source metadata.
 * @param[in]  user_data    A pointer to user-specific data.
 */
typedef gpointer (*NvDsMetaCopyFunc) (gpointer data, gpointer user_data);

/**
 * \brief  Defines the type of a callback to free metadata.
 *
 * The callback is passed a pointer to metadata created by NvDsMetaCopyFunc().
 * It frees the metadata and
 * any other resources that the metadata uses.
 *
 * @param[in]  data         A pointer to the metadata to be freed.
 * @param[in]  user_data    A pointer to user-specific data.
 */
typedef void (*NvDsMetaReleaseFunc) (gpointer data, gpointer user_data);

/**
 * Holds information about base metadata of a given metadata type.
 */
typedef struct _NvDsBaseMeta {
  /** Holds a pointer to batch_meta @ref NvDsBatchMeta. */
  struct _NvDsBatchMeta *batch_meta;
  /** Holds the metadata type of this metadata element. */
  NvDsMetaType meta_type;
  /** Holds a pointer to a user context. */
  void * uContext;
  /** Holds a user-defined copy function. The function is called to copy
   or transform metadata from one buffer to another. @a meta_data and
   @a user_data are passed as arguments. */
  NvDsMetaCopyFunc copy_func;
  /** Holds a user-defined release function. The function is called
   when @a meta_data is to be released. */
  NvDsMetaReleaseFunc release_func;
} NvDsBaseMeta;

/**
 * Holds user metadata.
 */
typedef struct _NvDsUserMeta {
  NvDsBaseMeta base_meta;
  /** Holds a pointer to user data to be attached.
   See the deepstream-user-metadata-test example for usage. */
  void *user_meta_data;
} NvDsUserMeta;

/**
 * Generates a unique user metadata type from a specified
 * string describing user-specific metadata.
 *
 * @param[in] meta_descriptor A pointer to a string
 * describing user-specific metadata. The string must
 * be in the format @a ORG_NAME.COMPONENT_NAME.METADATA_DESCRIPTION,
 * e.g. @c NVIDIA.NVINFER.TENSOR_METADATA.
 */
NvDsMetaType nvds_get_user_meta_type(gchar *meta_descriptor);

不在官方設計的Meta Data全部都歸類在NvDsUserMeta，它提供一個user_meta_data讓你附加任意資料，開發者要保證這筆資料的生命週期。建立NvDsUserMeta還需要去base_meta設定copy_func以及release_func，讓框架知道如何去複製(Deep Clone)及釋放你的user_meta_data，uContext則是讓你附加不屬於Meta Data的任意資料。

每個NvDsUserMeta都要設定base_meta.meta_type，務必使用nvds_get_user_meta_type這個函式來取得你的NvDsMetaType，參數meta_descriptor務必要遵循文件的格式ORG_NAME.COMPONENT_NAME.METADATA_DESCRIPTION，否則會出錯。你可以用macro描述你的NvDsMetaType

#define IRONMAN_DEEPSTREAM_META nvds_get_user_meta_type("IRONMAN.DEEPSTREAM.METADATA")

NvDsMetaCopyFunc和NvDsMetaReleaseFunc實作時要小心一個地方，就是user_data傳入的是NULL，data傳入的是NvDsUserMeta，所以你得從data才能拿到uContext。

結語

老實說，如果你的應用完全照著DeepStream設計好的方式開發，那麼你是根本不需要理會這些Meta Data的，元件基本上都幫你把資料處理好了。唯一比較需要注意的是如果你的模型要寫Custom Parse(後處理)，那就必須要了解一下那幾個ID欄位是做什麼的，畢竟要填對數值才會顯示正確的文字